import requests
from lxml import etree
import json
from xpinyin import Pinyin
import pymysql
import configparser
import uuid
import datetime
from pymysql import IntegrityError


class Test:
    # 读取配置文件   初始化数据库连接
    def __init__(self, province, city):
        config = configparser.ConfigParser()
        config.read("config.ini")
        host = config.get("dbconfig", "host")
        user = config.get("dbconfig", "user")
        password = config.get("dbconfig", "password")
        db = config.get("dbconfig", "db")
        port = int(config.get("dbconfig", "port"))
        self.createId = int(config.get("dbconfig", "createId"))
        self.updateId = int(config.get("dbconfig", "createId"))
        self.connect = pymysql.connect(host=host, user=user, password=password,
                                       db=db,
                                       port=port)
        self.cursor = self.connect.cursor()
        print("数据库连接成功")

        self.province = province
        self.city = city.replace("市", "")
        p = Pinyin()
        self.c = p.get_pinyin(self.city).replace("-", "")
        # print("c:>>>>>>", self.c)
        self.prefix = "https://yi.9939.com"
        self.start_url = "https://yi.9939.com/shop/{}/?page={}&per-page=50"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36"}
        pass

    # 爬虫获取网页数据
    def getData(self):
        i = 1
        while True:
            url = self.start_url.format(self.c, i)
            print("url:>>>>>", url)
            response_html = requests.get(url, headers=self.headers).text
            # print(response_html)
            html = etree.HTML(response_html)
            print("第{}页开始导入".format(i))
            lis = html.xpath('.//div[@class="comresult"]//div[@class="body"]/li')
            # print(len(lis))
            dataList = []
            for li in lis:
                data = {}
                href = li.xpath("./a/@href")[0]
                # print(href)
                final_url = self.prefix + href
                # print("最终拼接的url是:>>>>", final_url)
                response_detail = requests.get(final_url, headers=self.headers).text
                detail_html = etree.HTML(response_detail)
                detail_div = detail_html.xpath('.//div[@class="shoptopl"]')[0]
                data["c_store_name"] = detail_div.xpath('./p[2]/text()')[0]
                data["c_phone"] = detail_div.xpath('./p[4]/text()')[0]
                data["c_address"] = detail_div.xpath('./p[8]/text()')[0]
                dataList.append(data)
            self.dealData(dataList)
            next_page = html.xpath('.//li[@class="next"]/a/@href')
            if len(next_page) == 0:
                print("这是最后一页数据")
                break
            i += 1
        self.closeDB()

        # for n in range(33, 34):
        #     print("第{}页数据开始导入!".format(n + 1))
        #     url = self.start_url.format(self.c, n + 1)
        #     # print("url:>>>>", url)
        #     response_html = requests.get(url, headers=self.headers).text
        #     # print(response_html)
        #     html = etree.HTML(response_html)
        #     next_page = html.xpath('.//div[@class="comresult"]//li[@class="next disabled"]/@href')
        #     print("next_page:>>>", len(next_page))
        #     if next_page:
        #         break
        #     lis = html.xpath('.//div[@class="comresult"]//div[@class="body"]/li')
        #     # print(len(lis))
        #     dataList = []
        #     for li in lis:
        #         data = {}
        #         href = li.xpath("./a/@href")[0]
        #         # print(href)
        #         final_url = self.prefix + href
        #         # print("最终拼接的url是:>>>>", final_url)
        #         response_detail = requests.get(final_url, headers=self.headers).text
        #         detail_html = etree.HTML(response_detail)
        #         detail_div = detail_html.xpath('.//div[@class="shoptopl"]')[0]
        #         data["c_store_name"] = detail_div.xpath('./p[2]/text()')[0]
        #         data["c_phone"] = detail_div.xpath('./p[4]/text()')[0]
        #         data["c_address"] = detail_div.xpath('./p[8]/text()')[0]
        #         dataList.append(data)
        #     self.dealData(dataList)
        #
        # self.closeDB()

    # 处理数据
    def dealData(self, dataList):
        for data in dataList:
            data["c_id"] = str(uuid.uuid1()).replace("-", "")
            data["c_phone"] = str(data["c_phone"]).split("：")[1]
            store_name = data["c_store_name"]
            area = ""

            address = "-"
            if "：" in data["c_address"]:
                address = str(data["c_address"]).split("：")[1]
                data["c_address"] = address
            m = self.get_area_index(address, "市", 2)
            if "区" in address:
                end_index = address.index("区")
                start_index = end_index - 2
                area = address[start_index:end_index]
            elif "县" in address:
                end_index = address.index("县")
                start_index = end_index - 2
                area = address[start_index:end_index]
                if area == self.city:
                    area = ""
            elif m:
                start_index = m - 2
                area = address[start_index:m]
            #     如果地址里面没有匹配到area，就从药店名称里面进行匹配
            elif "市" in address:
                end_index = address.index("市")
                start_index = end_index - 2
                area = address[start_index:end_index]
            if area == "":
                n = self.get_area_index(store_name, "市", 2)
                if "区" in store_name:
                    end_index = store_name.index("区")
                    start_index = end_index - 2
                    area = store_name[start_index:end_index]
                elif "县" in store_name:
                    end_index = store_name.index("县")
                    start_index = end_index - 2
                    area = store_name[start_index:end_index]
                    if area == self.city:
                        area = ""
                elif n:
                    start_index = n - 2
                    area = store_name[start_index:n]
                else:
                    area = store_name[0:2]
                    if area == self.city:
                        area = ""

            data["c_province"] = "-"
            data["c_city"] = "-"
            data["c_area"] = ""
            province = self.province
            city = self.city
            with open("province.json", "r", encoding="UTF-8") as json_object:
                list = json.load(json_object)
                flag = False
                for i in list:
                    if province == i["label"]:
                        data["c_province"] = i["value"]
                        for j in i["children"]:
                            if city in j["label"]:
                                data["c_city"] = j["value"]
                                flag = True
                                if "children" in j.keys():
                                    for k in j["children"]:
                                        if area in k["label"]:
                                            data["c_area"] = k["value"]
                                            break
                            if flag:
                                break
                    if flag:
                        break
            data["n_open_type"] = 2
            data["n_ischain"] = 2
            if "连锁" in data["c_store_name"] or "分" in data["c_store_name"] or "路" in data["c_store_name"]:
                data["n_ischain"] = 1
            time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            data["dt_create_time"] = time
            data["dt_update_time"] = time
        print("处理后的数据是:>>>>")
        for i in dataList:
            print(i)
        self.importData(dataList)

    # 导入数据
    def importData(self, dataList):

        insert_sql = 'INSERT INTO t_nmd_mp_drug_store (c_id,c_store_name,c_province,c_city,c_area,c_phone,n_open_type,' \
                     'n_ischain,c_address,n_create_id,dt_create_time,n_update_id,dt_update_time) values(%s,%s,%s,%s,%s,' \
                     '%s,%s,%s,%s,%s,%s,%s,%s)'
        for data in dataList:
            try:
                # print("area:>>>", data["c_area"])
                self.cursor.execute(insert_sql, (data["c_id"], data['c_store_name'], data['c_province'], data['c_city'],
                                                 data['c_area'],
                                                 data['c_phone'], data['n_open_type'], data['n_ischain']
                                                 , data['c_address'], self.createId, data["dt_create_time"],
                                                 self.createId,
                                                 data['dt_update_time']))
                self.connect.commit()
                print(data["c_store_name"], "插入成功")
            except IntegrityError:
                print(data["c_store_name"], "已存在")

    # 关闭数据库连接
    def closeDB(self):
        self.connect.close()
        self.cursor.close()
        print("导入完成!")

    # 如果address中 "市" 出现两次，则获取"市"第二次出现的下标
    def get_area_index(self, address, city, num):
        count = address.count(city)
        if count > 1:
            i = 0
            index = -1
            while i < num:
                index = address.find(city, index + 1)
                i += 1
            return index
        else:
            return None


if __name__ == '__main__':
    province = input("请输入省份")
    city = input("请输入城市")
    test = Test(province, city)
    test.getData()
    pass
